from queue import Queue
import requests
import threading
import time
import os
import redis
import json
import sys
from ossupload import upload
import re
import random
import platform
import resource

thread_count = 16
copies_count = thread_count * 4
requests.packages.urllib3.disable_warnings()
# 查看是linux还是win
sys_platform = platform.platform().lower()
sys_status = 2 if "windows" in sys_platform else 1
dir = '/data/down/'
redis_db = redis.Redis(host='98.82.254.89', port=6379, db=5, password='953058009',
                       decode_responses=True, charset='UTF-8', encoding='UTF-8')


# 国外下载
class DownloadThread(threading.Thread):
    def __init__(self, bytes_queue: Queue, url):
        super().__init__(daemon=True)
        self.bytes_queue = bytes_queue
        self.url = url

    def run(self):
        while not self.bytes_queue.empty():
            bytes_range = self.bytes_queue.get()
            headers = {
                "User-Agent": get_ua(),
                "Referer": 'https://www.xitongzu.com',
                "Range": "bytes={}".format(bytes_range[1])
            }
            try:
                response = requests.get(self.url, headers=headers, verify=False, timeout=30)
                with open("files/{}.tmp".format(bytes_range[0]), "wb") as f:
                    f.write(response.content)
            except Exception:
                return False


def get_file_size(url) -> int:
    try:
        headers = {
            "Referer": 'https://www.xitongzu.com',
            "User-Agent": get_ua()
        }
        response = requests.head(url, headers=headers, verify=False, timeout=30)
        file_length = int(response.headers['Content-Length'])
    except Exception:
        file_length = 0
    return file_length


def get_thread_download(file_length) -> list:
    bytes = Queue(copies_count)
    start_bytes = -1
    for i in range(copies_count):
        bytes_size = int(file_length / copies_count) * i
        if i == copies_count - 1: bytes_size = file_length
        bytes_length = "{}-{}".format(start_bytes + 1, bytes_size)
        bytes.put([i, bytes_length])
        start_bytes = bytes_size
    return bytes


def create_threading(bytes_queue, url):
    thread_list = []
    for i in range(thread_count):
        thread = DownloadThread(bytes_queue, url)
        thread.start()
        thread_list.append(thread)

    for thread in thread_list:
        thread.join()


def composite_file(filename):
    if os.path.isfile(filename): os.remove(filename)
    # 防止文件不存在
    try:
        with open(filename, "ab") as f:
            for i in range(copies_count):
                with open("files/{}.tmp".format(i), "rb") as bytes_f:
                    f.write(bytes_f.read())

        for i in os.listdir("files"):
            os.remove("files/{}".format(i))
    except Exception:
        return False


def hum_convert(value):
    units = ["B", "KB", "MB", "GB", "TB", "PB"]
    size = 1024.0
    for i in range(len(units)):
        if (value / size) < 1:
            return "%.2f%s" % (value, units[i])
        value = value / size


# 获取下载的文件数据
def get_redis_data(name=''):
    list = redis_db.get(name)
    return json.loads(list)


# 查看是否已下载过
def is_download(filename):
    title = 'down_27_win_log'
    status = 0
    if not redis_db.hexists(title, filename):
        redis_db.hset(title, filename, 0)
        status = 1
    return status


# 获取文件夹下所有子级的文件名称
def get_file():
    dirname = dir
    if not os.path.exists(dirname):
        print('该文件夹不存在')
        sys.exit()

    file_list = []
    # 递归获取子级
    for root, dirs, files in os.walk(dirname):
        for file in files:
            path = os.path.join(root, file)
            file_list.append(path)
    return file_list


# 验证豌豆荚
def checkwandoujia(download_url):
    url_list = [
        'wandoujia.com',
        'mobgslb.tbcache.com'
    ]
    ip = getOutterIP()
    info = redis_db.get('wandoujia' + ip)
    current_time = int(time.time())
    for url in url_list:
        if url in download_url:
            redis_db.set('wandoujia' + ip, current_time)
            if info and current_time - int(info) < 180:
                time.sleep(current_time - int(info))
                return True
    return True


def getOutterIP():
    ip = ''
    try:
        res = requests.get('https://myip.ipip.net', timeout=5).text
        ip = re.findall(r'(\d+\.\d+\.\d+\.\d+)', res)
        ip = ip[0] if ip else ''
    except:
        pass
    return ip


# 随机获得ua
def get_ua():
    first_num = random.randint(55, 62)
    third_num = random.randint(0, 3200)
    fourth_num = random.randint(0, 140)
    os_type = [
        '(Windows NT 6.1; WOW64)', '(Windows NT 10.0; WOW64)', '(X11; Linux x86_64)',
        '(Macintosh; Intel Mac OS X 10_12_6)'
    ]
    chrome_version = 'Chrome/{}.0.{}.{}'.format(first_num, third_num, fourth_num)

    ua = ' '.join(['Mozilla/5.0', random.choice(os_type), 'AppleWebKit/537.36',
                   '(KHTML, like Gecko)', chrome_version, 'Safari/537.36']
                  )
    return ua


# 去下载
def download(download_url, filename, num):
    startt = time.time()
    file_length = get_file_size(download_url)
    print('文件大小：' + str(hum_convert(file_length)))
    status = 0
    if file_length == 0:
        print(f"第{num}个文件->" + filename + "<-下载执行失败")
    else:
        copies_queue = get_thread_download(file_length)
        create_threading(copies_queue, download_url)
        composite_file(filename)
        # 检查大小是否一致
        size = os.path.getsize(filename)
        if size != file_length:
            os.remove(filename)
            print(f"第{num}个文件->{filename}<-下载大小不一致已删除")
            status = 2
        else:
            end = round(time.time() - startt, 2)
            print(f"第{num}个文件->{filename}<-下载用时" + str(end) + "秒")
            status = 1
    return status


# 不分片下载
def down(url, save_name):
    print('开始下载第二次')
    try:
        startt = time.time()
        filesize = get_file_size(url)
        headers = {
            "Referer": 'https://www.xitongzu.com',
            "User-Agnt": get_ua()
        }
        file = requests.get(url, allow_redirects=True, headers=headers, timeout=60)
        with open(save_name, 'wb') as f:
            f.write(file.content)
            f.close
        size = os.path.getsize(save_name)
        status = 1
        if filesize != size:
            os.remove(save_name)
            status = 2
            print('第二次下载大小不一致已删除')
        else:
            end = round(time.time() - startt, 2)
            print('第二次下载完成,用时' + str(end) + "秒")
    except:
        status = 2
        pass
    return status


# 去请求php分析
def analyes_demo(filename, system, file_type, is_exist):
    url = "http://38.180.6.8/api/pool/analyes?filename=" + filename + "&system=" + str(
        system) + "&type=" + str(file_type) + "&is_exist=" + str(is_exist)
    requests.packages.urllib3.disable_warnings()
    response = requests.get(url=url, verify=False, timeout=180)
    res = json.loads(response.text)
    print(res['msg'])
    return res


def analyes(filename, system, file_type, is_exist):
    url = f"http://38.180.6.8/api/pool/analyes?filename={filename}&system={system}&type={file_type}&is_exist={is_exist}"

    # 忽略 SSL 警告
    requests.packages.urllib3.disable_warnings()
    try:
        # 发送请求
        response = requests.get(url=url, verify=False, timeout=120)
        # 检查响应状态码
        if response.status_code != 200:
            print(f"Request failed with status code {response.status_code}")
            return {"code": 0}
        # 如果响应内容为空，提前返回
        if not response.text.strip():
            print("Response text is empty.")
            return {"code": 0}

        # 尝试解析 JSON 数据
        try:
            res = json.loads(response.text)
            print(res['msg'])
            return res
        except json.JSONDecodeError:
            print("Failed to decode JSON. The response might not be valid JSON.")
            print("Response text:", response.text)
            return {"code": 0}

    except requests.exceptions.RequestException as e:
        print(f"Request failed: {e}")
        return {"code": 0}


# 下载失败通知
def downerror(filename, system):
    url = "http://38.180.6.8/api/pool/downerror?filename=" + filename + "&status=" + str(
        sys_status) + "&system=" + str(system)
    requests.packages.urllib3.disable_warnings()
    response = requests.get(url=url, verify=False)
    res = json.loads(response.text)
    print(res['msg'])
    return res


def set_limits():
    # 设置最大文件描述符数
    resource.setrlimit(resource.RLIMIT_NOFILE, (65535, 65535))
    # 设置最大进程数
    resource.setrlimit(resource.RLIMIT_NPROC, (4096, 8192))
    # 设置最大内存使用
    resource.setrlimit(resource.RLIMIT_AS, (2 * 1024 * 1024 * 1024, 2 * 1024 * 1024 * 1024))
    # 设置最大堆栈大小
    resource.setrlimit(resource.RLIMIT_STACK, (1024 * 1024 * 8, 1024 * 1024 * 16))


def main():
    set_limits()
    # 开始下载
    rname = 'analyse_linux' if sys_status == 1 else 'analyse_win'
    redis_db.set(rname + '_log', 1)
    # 找出要下载的数据
    num = 0
    upload_num = 0
    for i in range(1, 100000):
        detail = redis_db.rpop(rname)
        if detail:
            num += 1
            print('')
            print(f"开始执行第{num}个下载")
            detail = json.loads(detail)
            for v in detail:
                # 如果无需下载
                if v['is_exist'] == 1:
                    print(f"该文件是oss数据无需下载")
                    analyes_res = analyes(v['filename'], v['system'], v['type'], v['is_exist'])
                    if analyes_res['code'] == 1:
                        print(f"已分析完成")
                    else:
                        print(f"分析出错")
                else:
                    filename = dir + v['filename']
                    system = '安卓' if v['system'] == 1 else 'pc'
                    print(f"开始执行{system}下载")
                    # 去下载
                    res = download(v['download_url'], filename, num)
                    # 去开启第二遍下载
                    if res == 2:
                        try:
                            res = down(v['download_url'], filename)
                            if res == 1:
                                file_size = os.path.getsize(filename)
                                if file_size < 800:
                                    os.remove(filename)
                                    res = 2
                        except:
                            res = 2
                    # 如果太小则删除
                    if res == 1:
                        # 去分析
                        analyes_res = analyes(filename, v['system'], v['type'], v['is_exist'])
                        if analyes_res['code'] == 1:
                            if int(v['is_create_url']) > 0:
                                # 去上传
                                upload_num += 1
                                upload_res = upload(analyes_res['data']['path'], upload_num, v['type'],
                                       analyes_res['data']['file_name'])
                                if upload_res == False:
                                    downerror(filename, v['system'])
                            else:
                                os.remove(analyes_res['data']['path'])
                        else:
                            # 失败去删除
                            os.remove(filename)

                    else:
                        # 通知失败
                        downerror(filename, v['system'])
                        # 如果是windows只要错一次就不继续往下执行
                        if sys_status == 2:
                            break
        else:
            redis_db.set(rname + '_log', 0)
            break


if __name__ == '__main__':
    main()
